{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 逻辑回归\n",
    "- 线性决策边界的逻辑回归\n",
    "- 非线性决策边界的逻辑回归\n",
    "- 正则化后的逻辑回归算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<frozen importlib._bootstrap>:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "<frozen importlib._bootstrap>:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "# 优化函数，用于最小化损失函数\n",
    "from scipy.optimize import minimize\n",
    "# 数据预处理，PolynomialFeatures用于构造特征\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "\n",
    "pd.set_option('display.notebook_repr_html', False)\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', 150)\n",
    "pd.set_option('display.max_seq_items', None)\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "import seaborn as sns\n",
    "sns.set_context('notebook')\n",
    "sns.set_style('white')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据读取函数\n",
    "def loadData(file, delimeter):\n",
    "    data = np.loadtxt(file, delimiter=delimeter)\n",
    "    print('Dimension=>', data.shape)\n",
    "    # 输出前五行\n",
    "    print(data[1:6, :])\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据可视化函数\n",
    "def plotData(data, label_x, label_y, label_pos, label_neg, axes=None):\n",
    "    # 划分正负样本\n",
    "    pos = data[:, 2] == 1\n",
    "    neg = data[:, 2] == 0\n",
    "    \n",
    "    if axes == None:\n",
    "        axes = plt.gca()\n",
    "    # data[pos] 过滤出数据中的正样本，这种写法不太理解？？？\n",
    "    axes.scatter(data[pos][:,0], data[pos][:,1], marker='+', c='k', s=60, label=label_pos)\n",
    "    axes.scatter(data[neg][:,0], data[neg][:,1], c='y', s=60, label=label_neg)\n",
    "    axes.set_xlabel(label_x)\n",
    "    axes.set_ylabel(label_y)\n",
    "    axes.legend(frameon=True, fancybox=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dimension=> (100, 3)\n",
      "[[30.28671077 43.89499752  0.        ]\n",
      " [35.84740877 72.90219803  0.        ]\n",
      " [60.18259939 86.3085521   1.        ]\n",
      " [79.03273605 75.34437644  1.        ]\n",
      " [45.08327748 56.31637178  0.        ]]\n"
     ]
    }
   ],
   "source": [
    "data = loadData('data1.txt', ',')\n",
    "X = np.c_[np.ones((data.shape[0], 1)), data[:, 0:2]]\n",
    "Y = np.c_[data[:, 2]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 线性决策边界\n",
    "> 通过两门考试的成绩判断这个人是否通过测试，是则为 1，否则为 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEJCAYAAAB/pOvWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3df1xUdb4/8NeACCiWoYyQIGm7aldb9eKP5oqC1hKgCIGV6DXFdcV0KdFbmpqaVwONlW/uWkndq+VSlr8uXK5L10S4KmahoQ83xR6lBFYzI+YPUAaYOd8/iEkcYPgxc37MeT3/cs6MnPccOJ/35/fRCIIggIiI6C5uUgdARETyw+RAREQ2mByIiMgGkwMREdlgciAiIhvdpA6gq2pra3Hu3Dn4+fnB3d1d6nCIiBTBbDbDaDRi+PDh8PLysnlf8cnh3LlzmDVrltRhEBEpUnZ2NkaPHm1zXPHJwc/PD0DjF/T395c4GiIiZfjpp58wa9Ysaxl6L8Unh6auJH9/fwQGBkocDRGRsrTWHe/UAenq6mpMnToVlZWVAIDi4mLExMQgIiICmZmZ1s+dP38e8fHxePLJJ7Fq1So0NDQ4MywiIrLDacnhzJkzSExMxOXLlwE0DhyvXLkSb731Fg4ePIhz586hqKgIAPDSSy9hzZo1+PTTTyEIAj755BNnhUVERO3gtOTwySefYO3atdBqtQCAs2fPIjg4GEFBQejWrRtiYmKQn5+PK1euoLa2FiNHjgQAxMfHIz8/31lhERFROzhtzGHjxo3NXhsMhmYDH1qtFnq93ua4n58f9Hq9s8JyGEGwwGD4CBUVmTCZKuDpGYSgoFRotYnQaLh8hIiUTbQBaYvFAo1GY30tCAI0Gk2rx+VMECw4dy4eP//8GSyWGgBAfb0BZWXJMBr3YtiwfUwQRKRoopVg/v7+MBqN1tdGoxFardbm+NWrV61dUXJlMHzULDE0sVhqcO3aIRgMuyWKjIjIMURLDiNGjMClS5dQXl4Os9mMvLw8TJw4Ef3794enpydOnToFAMjJycHEiRPFCqtTKioybRJDE4ulBhUVW0SOiNQgPDwc4eHhUodBKiFat5KnpyfS09ORkpICk8mEsLAwREZGAgAyMjKwevVqVFdXY9iwYXjuuefECqtTTKYKO+9XOvX8TQVEYWGhU89DROrl9ORQUFBg/bdOp0Nubq7NZ4YOHYq9e/c6OxSH8fQMQn29oY33uRiPiJRN8SukpRAUlIqysuQWu5bc3HoiKGipBFGRs0nRYru7G6lpXdDdx+TUemSL1rUwOXSCVpsIg2GPzaC0m1tP+Pr+HlrtDIefU0mFhKtwhcLOFb4DSYPJoRM0GjcMH74fBsNuVFRsgclUCU/PQAQFLYVWO4PTWMlh7i7UWdCTmJgc7GhrsVu/fjPRr99MUeJgISGN0tJS6/Vmi80WW7Sui8mhDVzspj4tFXalpaUSRdM5LLDJEZgc2tCexW5itRxIOk37fpWWlmLkyJGSFa5yLNTZonVdTA5taM9iNymSA28+52mrsFPKAjQW2OQI7BNpg9SL3YiIpMKWQxu42I3uxtp323h9XAuTQxu42E3dXKGwc4XvQNJgt1IbtNpEPPDAE3Bz69nsuDMXuxERyQFbDm3gYjeSOw44k7MwOdih0biJutiNiEgOWPUlIqfjsyiUhy0HIoXhCmgSA1sORERkgy0HIoVRygpotnCUjS0HIhKN0jYxVDO2HIioGUe1Ru5t4Ui9cSF1DJMDkYKxoCVnkSQ5ZGVlYd++fejevTuio6Px/PPPo7i4GGlpaTCZTIiKikJqaqoUoRGRAzW1QkpLS3Hjxg0UFRVx3EEhRE8OxcXF+O///m/s27cP3t7eWLx4MXJzc7Flyxbs2rULAQEBSE5ORlFREcLCwsQOj0iVevfuDaDx2RXOGDy+++eSMoieHL7++muEhobCx8cHADBhwgTs2bMHwcHBCAoKAgDExMQgPz+fyYFkPRuH7FPKzCqyJXpyGDZsGF5//XUkJyfD29sbBQUFOH36NKKioqyf0Wq10Ov1YodGpFpNT7srLCxkId4GNV0b0ZODTqdDfHw8Zs+ejd69e0On0+HYsWPQaDTWzwiC0Ow1SU9NN4VatLYOgdNNCZAgOVRXVyMiIgJJSUkAgPfeew9jx46F0Wi0fsZoNEKr1YodGskEF0+5Jv7elEX05FBZWYnly5dj3759uHPnDvbu3YuNGzdi6dKlKC8vR2BgIPLy8pCQkCB2aESqwvGA9lFrZUX05DB06FBERERg2rRpMJvNmDt3LkJCQpCeno6UlBSYTCaEhYUhMjJS7NDoHlLdFCy05EPN11/tu8hKss5h8eLFWLx4cbNjOp0Oubm5UoTjctR8QxM5mlorK1whTa1S602hRvy9yo/U9xyTA8mamDeG1DejXKi1jx2w/93VhMnBRaj5hiYSi5ruIyYHahc13RRyIXVLRo0L4qTuSpVTJY/JwUVI/UetVHK6GYnkhMmBSOVYmZAPOVXymBxI1eR0M94dAyBuS6a0tNR6HrUPxDZRe7JkcnBBav+jJqKuY3IgkhGxWjIttVCa3H///XycpwxIff2ZHIh+IfXNSCQnTA5EKtRWC0XtYw3UiMmBSKakXN9A5CZ1AEREJD9sORCpHFsK1BK2HIiIyAaTAxER2WByICIiG0wORA4UHh7OqaDkEpgciIjIBpMDERHZkGQqa05ODrKysgAAEydOxPLly1FcXIy0tDSYTCZERUUhNTVVitCIOozPhCBXJHrL4c6dO9i4cSN27dqFnJwclJSUoKCgACtXrsRbb72FgwcP4ty5czabgRERkXhEbzmYzWZYLBbcuXMHPXr0QENDA3x8fBAcHIygoCAAQExMDPLz8xEWFiZ2eEQdJrdnQhA5gujJwcfHBy+++CKioqLg7e2NMWPGwGAwwM/Pz/oZrVYLvV4vdmiKJggWGAwfoaIiEyZTBTw9gxAUlAqtNhEaDYeWiKhjRC81Lly4gH379uHIkSM4evQo3NzccPnyZWg0GutnBEFo9praJggWnDsXj7KyZFRXn0J9vQHV1adQVpaMf/wjAYJgkTpEEhGn05IjiJ4cjh07Bp1Ohz59+qB79+6Ij4/HyZMnYTQarZ8xGo3QarVih9ZpgmCBXp+NkpLROH68H0pKRkOvzxatUDYYPsLPP38Gi6Wm2XGLpQbXrh2CwbBblDiIyHWInhyGDh2K4uJi3L59G4IgoKCgACNGjMClS5dQXl4Os9mMvLw8TJw4UezQOkUOtfaKikybxNDEYqlBRcUWp8dARK5F9DGH0NBQfP3114iPj4eHhwceffRRpKSkYPz48UhJSYHJZEJYWBgiIyPFDq1T2lNr79dvplNjMJkq7Lxf6dTzk/Q4nZYcTZJ1DgsWLMCCBQuaHdPpdMjNzZUinC5pT63d2cnB0zMI9fWGNt4PdOr51Y4FM7VE6TPX+DyHLpJDrT0oKBVlZcktJik3t54IClrq9BhIWpxOS47G5NBFcqi1a7WJMBj22HRvubn1hK/v76HVznB6DGrGgtkxeO3khcmhi5xZa2/v2gWNxg3Dh++HwbAbFRVbYDJVwtMzEEFBS6HVzuA6h3uwECJncaUuRiaHLnJWrb1pFtTdP7e+3oCysmQYjXsxbNg+mwTRr99Mp49vdBQLYvHxWpMj2E0ONTU1yMjIwHfffYc333wTW7ZswfLly9GzZ08x4pO1ppq9yVQJjcYdbm49oNF0g5fXbzBgwLIu1drlMAuKOo4Fc8cqBK5U0wZcq4vRbnLYsGEDtFotqqqq4OnpierqaqxZswZ//vOfxYhPtlqq2QONLQZv7wFd7s6RwywoV+JqhRCRs9lNDufPn0daWhqKiorg7e2NjIwMTJ06VYzYZM3ZNXs5zILqLBbE1F6uVNN2NXaTg5tb89qv2Wy2OaZGzq7Zy2EWlCthIeRcbVUISktLMXLkSNVdb6V/X7vJYcyYMXjjjTdQW1uLo0ePIjs7G+PGjRMjNllzds1eyWsXWBATKZ/d5PBv//ZvyMrKQq9evZCZmYkJEyZg0aJFYsQma86u2XPtAilJWxWC9u4QywqEvNhNDlu3bsWyZcuwePFiMeJRDGfX7Ll2wXnaKoTY0nGM0tJS67XkuJMy2U0OhYWFWLZsmRixKEp7avZdfQCPXNcudITSCwEmC1Iru8khMDAQ8+bNwz//8z83W9uQlJTk1MDkzl7NHkCHFrERuYp7EykTrPM489raTQ69e/cGAFy5csXhJ1e6tmr2en02F7EpRHtm2hCpjd3kkJaWBqAxOTQ0NCA4ONjpQbkCLmJTttLSUgDAjRs3UFRUxP5yUh27yaG8vByLFi2CwWCAxWLBAw88gO3bt+Phhx8WIz7FUvIiNrW5d6bN3a2FppYEdR6TqWOJtcjUbnJYv3495s+fj6eeegoAsG/fPrz22mv44IMPHBKAq3L0VNeuDm5T+929YIv95aRWdpNDVVWVNTEAQEJCAnbu3OnMmFyCI6e6dnSHViJyXWItMrVbopjNZly/ft36+tq1aw4PwhVptYl44IEn4ObWfPfazixia88+TuQYhYWFbCXYER4e3u6FbaRcdlsO//qv/4pnn30WUVFR0Gg0OHjwIObMmSNGbIrmyEVsHNyWDqdlklrZTQ7PPvssgoODcfToUVgsFqxbtw46na7TJ9yzZw/+9re/WV9XVlYiNjYWTzzxBNLS0mAymRAVFYXU1NROn0MuHLWIjYPb5CxqT3ZK//7OjNtuctDr9cjPz8e6devw3XffISMjA7/5zW/g5+fXqRM+/fTTePrppwEA33zzDRYvXow//vGPSExMxK5duxAQEIDk5GQUFRUhLCysU+dwNa66Q6vSb0w14Tbs6mM3OSxfvhyTJ08GAPTv3x9jx47FypUr8e6773b55OvWrUNqaioqKioQHByMoKAgAEBMTAzy8/OZHH6h5B1aXQELxtY1rQch12M3Ofz888947rnnAACenp6YO3cu/uu//qvLJy4uLkZtbS2ioqKQl5fXrCWi1Wqh1+u7fA5XwR1ayZE6k+xamyGjxIFpJST7tlrVYrW47SYHs9kMvV6Pfv36AQCuXr0KQRC6fOLdu3db92eyWCzQaDTW9wRBaPZa7Vxph1Yl3Jj34vMpSI3sJoe5c+ciLi4OEyZMgEajQXFxMV5++eUunbSurg5ffvkl0tPTAQD+/v4wGo3W941GI7RabZfO4WpcYYdWkoeuJrumrqTw8HDFJPi7Mdm3j93kMH36dAwfPhyff/453N3d8Yc//AGDBw/u0knLysrw0EMPoUePHgCAESNG4NKlSygvL0dgYCDy8vKQkJDQpXOQPPHGVD5uROgc7d0AUqyEbDc5AEDPnj0xd+5cHDlyBIcOHUJAQAB69erV6ZNWVFTA39/f+trT0xPp6elISUmByWRCWFgYIiMjO/3ziZyFiYwJXi3sJoc1a9YAAObMmYNXX30VEyZMwMqVK/GXv/yl0yeNjo5GdHR0s2M6nQ65ubmd/plE1HFqL9Tl9P3bm3RlMyB97tw57N27F1lZWXjqqaewbNkyxMfHOzUoUgc53ZhSYw2c5MZuchAEAW5ubjh+/DgWLlwIAKitrXV6YOQ83OGVHIXJzHXZTQ4DBgzAH//4R1RWVmLs2LFYtmwZhg4dKkZs5ATc4ZVI/tpKumIl5HY9Ce7QoUMICQmBh4cHRo8ejbi4ODFiIydozw6vnC4rDiWu+XBlzujaU3J3od0qYo8ePRAbG4vAwMb9exITE+Ht7e30wMg52rPDKxFRu6aykjjEGAtwlR1elVwja8IpoSRnTA4yIdZYgKvu8ErUGc7o2nOV7kK7pU1DQ4PNsRs3bjglGDUT62lvQUGpNk+na8IdXomoSasth3PnziElJQVXr17F448/jg0bNsDHxwdA435LBw4cEC1INRDraW9K3uHVVWpkLVFy7ErmjK49V+kubLXlsHHjRqxbtw6FhYXo1q0b5s+fj7q6OgBwyK6s1JxYYwFNO7wOGZIFH58QeHj0g49PCIYMyeI0ViKyarXlUFtba33YTkZGBl544QW88sor+POf/yxacGoi5liA1Du8dnbg3VVqZERK0GpysFgsqKqqQp8+fQAAmzZtwowZM7Bt2zY+a8EJ1PK0Ny7CI7lyRkVDyZWXVu/CefPmIS4uztq36+3tjbfffhv79+/HxYsXRQtQLbTaRDzwwBM2g8VKGAvoCLEG3omoa1ptOcTGxuJ3v/sdunfvbj324IMPIjc3F9nZ2aIEpyau9LS3tjhq4F3JNTIiJWhzncPAgQNtjvXs2RMLFixwWkBqJvVYgBhcZREekatzjeooKYanZ5Cd97kIj0gOmBxIVFyE96vw8PBm6zSI2kOsvxsmBxJVawPvAKDReEAQzBAEiwSREdHd7O6tdPLkSWRlZdlsmbF3716nBaU0fHhO+zUNvOv1H+Kbb/4Es/kmgMZFlWbzdVy8+DyuXt3PKa1EErObHFavXo3Zs2djwIABYsSjOJy333EajRs0Gg0EoQFNiaGJqz9XwpW3ACHnkeLvxm5y6NOnD5577jmHnrSgoAB//etfcefOHYwfPx6rV69GcXEx0tLSYDKZEBUVhdTUVIee01n48JxGHW09ibWXFBF1jt3kMHnyZGRnZ2PChAno1u3Xjz/44IOdOmFFRQXWrl2LPXv2oE+fPpgzZw6Kioqwdu1a7Nq1CwEBAUhOTkZRUZF1+w45YyHXudaTs6e0ynV7DW4BQp0hxd+N3eTw888/Y8uWLc2e/qbRaHD69OlOnfDQoUOIjo6Gv78/ACAzMxPl5eUIDg5GUFDjNMeYmBjk5+crIjlw3n7nWk98rgSRvNlNDkeOHMGxY8fQt29fh5ywvLwcHh4eWLhwIX788UeEh4fjt7/9Lfz8/Kyf0Wq10Ov1Djmfs7GQ61zrSS17SREpVbvGHHx9fR12QrPZjJKSEuzatQs9evTA888/Dy8vr2ab+QmCoJjN/VjIda715IznSihtsFdu8ZAyiPV3Yzc5DB48GDNnzsSkSZOa7bOUlJTUqRP27dsXOp3OmnCeeOIJ5Ofnw93d3foZo9EIrVbbqZ8vNiU/PMdROtN6UsteUkRKZTc51NbWYuDAgbh8+bJDTjhp0iQsX74cN2/eRM+ePXH06FFERkYiKysL5eXlCAwMRF5eHhISEhxyPmdjIdf51pOj95LiYC+R49hNDmlpaQ494YgRIzB//nzMnDkT9fX1GD9+PBITEzFo0CCkpKTAZDIhLCwMkZGRDj2vM6lhw7y2sPVE5HrsJoevvvoKWVlZuH37NgRBgMViQWVlZZdqZNOnT8f06dObHdPpdMjNze30zyTpsPVEUmIr0TnatUI6NjYWn376KWbMmIHDhw8jIiJCjNhIQeTWemJBQdQ1dpODRqPBggUL8PPPP2PQoEGIiYlRzHgAtY17QpFSsHUgPrvJoWfPxt0zBwwYgG+++QYhISFwc2PBoXTcE4qUTGnTlpXI7t3/6KOPYsmSJXjsscfwn//5n0hPT2+2jQYpE5/lTERtsVvKr1q1CmfOnMHAgQOxcuVKFBcXY8uWLWLERk7EPaFI7trbOmCXk3PYbTnk5uZi5MiRABp/CS+//DL27dvn9MDIubgnFBG1xW7LYfv27Th9+jRWrVoFg8GA1NRUh26nQdLgnlAkd1zUKC27LYe9e/eioaEBCQkJmDVrFp555hls375djNjIifgsZ3IVhYWFTBpOYDc5aDQadO/eHXfu3IHFYlHMhnjUttae5cxVzUQEtKNbadq0aRg5ciRycnJw9epVLFu2DJ999hneeecdMeIjJ+GqZlIStgzEZzc5LF68GHFxcQAa1zx89NFHyMzMdHpg5HxyW9VMRPLRavXwhx9+AABrYmji4eGBxx57zLlREZFVeHh4symcRGJoNTksXrzY+u+UlJRm77HlQETk2lrtVhIEwfrvioqKVt8jcjY57AElhxiIxNRqcrh7VtK9M5Q4Y4maOLvQlMMeUFLEwL2DSGqt/kWzdUD2NBWaZWXJqK4+hfp6A6qrT6GsLBn/+EcCBMHS5XPIYQ8oOcRAJLZWWw4WiwU3btyAIAgwm83WfwOA2WwWLUCSr/YUml2dCSWHPaCkiIGrg0lqrSaHixcv4rHHHrMmhHHjxlnfY7cSAeIUmnLYA0oOMRCJrdXkcOHCBTHjIAUSo9CUwx5QcoiBSGycZkGd5ukZZOf9rheaUu8BJQgW3HffWLR2q4gRA/cOIilI8tSe2bNn49q1a9aHBq1fvx41NTVIS0uDyWRCVFQUUlNTnXZ+Tkt0jKCgVJSVJbfYteTm1hP33TcOJSWju3SNtdpEGAx7bMY2xNgDqmnA/dq1QwBsB9e5DxW5MtGTgyAIuHz5Mo4cOWJNDrW1tYiMjMSuXbsQEBCA5ORkFBUVISwszAnnl35qpKtoq+B2d++FH3/cCUG4DaDz11jKPaCaBtybvkNzbvD3n4Pf/vYv/HshlyR6cvjuu+8AAPPmzcP169fxzDPPYPDgwQgODkZQUGM3RUxMDPLz852SHMSYYaMWrRXc9903rlliaNLZayzVHlBtDbgDFty8eZKJgVyW6H/ZN2/ehE6nw7Zt27Bz507s3r0bP/zwA/z8/Kyf0Wq10Ov1Tjl/e2bYUPs1FdyjR5dg/PifMHp0CW7ePNlKbVtZ15izlEjNRG85jBo1CqNGjbK+nj59OrZu3YqQkBDrMUEQnDZdlje889m7xjU15yAIFtnXujlLSXpc4yEd0e/OkpISnDhxwvpaEAT0798fRqPResxoNEKr1Trl/GLMsFE7e9dYEOoVsapY6plSRFISPTncunULmzdvhslkQnV1NQ4cOIClS5fi0qVLKC8vh9lsRl5eHiZOnOiU8/OGd76goFS0/adlUUTXEp+WZ4vbh6uH6N1KkyZNwpkzZxAXFweLxYKZM2di1KhRSE9PR0pKCkwmE8LCwhAZGemU80s5NVIttNpEXLgwD4JQ1+pnlNB9x6flSYObDsqDJOsclixZgiVLljQ7ptPpkJub6/RzO/uG5xqKxmvco8dw1NScbvUz9fVVKCkZLftrw6flkVpJkhyk5qwbnmsofjVgwNJWF8g1arDu4Kq2a9MaqSsWrQ3+il2T56aD8qDuu9HBuLXzr1rrr7+XGq9NS8TY/pyoI1TZcnAWOWwvLRd3d999//2fUVNTipa2oADUd21aIufFmazJqxOTgwNxDUVzGo0btNoZMBg+aXP8AVDftbmXVBULuQ/+Sn1+R1FiUmW3kgNxDYWtphqxPWq8NndjxUIeOFX3V2w5OJC9XUrVuIai7f2JGqn12txNqtXYHe0y6mzNV4k1Z7VjcnAgrqGwZa9GDEC11+ZurFi4Hrl32dnD5NAFLU09DAx8EX5+01FZ+f+4aAr2a8ReXgM5jRWsWEhJ6YW4s6g2OXR1TnlraxouXnwevr6/R0jIF6ov8AD7NeKBAzfwOkEeq7EdXQiqvdBV+iwvVSYHRyxWk/PUQzlhjbj9uBrbPmcUsq0V4mofmFZlcuhowd5SK6O+voprGtpBDjVikobSa85qp8rk0JE55a21Muzh1MNfsUbcOqm3zJADNVwDJSZFVSaHjswpb62VYY/a5+0rjRQFFPfiat81mDRpsvXzYo1dhIeHq3Kc5G6qTA4dmVPennn69+LUQ2WRqpBW07hVa4Uq9yOTL1Umh47MKW/PPP17/z8HWpVFqkKae3G17xoUFpZYjzl77ILjJL9y7TZrKzryhC97W2J4eQ2Ej08IPDz6wccnBEOGZKmiO8CVtKeAcgY5bJkhCBbo9dkoKRmN48f7oaRkNPT6bNF2gZXDNaCWqbLl0JEZNO2Zp+/qtTtXJ3YB1TS+YTZXt/k5Z49byWHMQ6ptQ8g+VSYHoP0zaDhP3/WJWUA1L5Bvt/o5Mcat5DDm0dFtQ8Ts4lFrd1IT9n3Y0dTKGDIki91HLiooKLXVhxI5upBuz+w3sSoeUnWn3c3P71m4u/dq8b1u3e6Dn98zTo+BWqbalkNHcJ6+axOzdWhv9pubWw8MGZIlygJBOfT3G40fo6HhZovv1dffgNH4Ce87iUha7d20aRNWrFgBACguLkZMTAwiIiKQmZkpZVikMmK2Du0VyO7uvdCv30xRWqRyeP5IRUUmBKHl7jVBuC1K64VaJlnL4cSJEzhw4ADCw8NRW1uLlStXYteuXQgICEBycjKKiooQFhbWpXPU19ejsrIStbW1DopaGby8vBAYGAgPDw+pQ3E6Ry1eE6t1KKcBWDlsEy6H1gu1TJLkcP36dWRmZmLhwoW4cOECzp49i+DgYAQFNdZkYmJikJ+f3+XkUFlZiV69euGhhx6CRqNxROiyJwgCqqqqUFlZiYEDB0odjlPJYbZNR8mhQG4ih8kWckqW1Jwkd86aNWuQmpqK++67DwBgMBjg5+dnfV+r1UKv13f5PLW1tejTp49qEgMAaDQa9OnTRxWtJSWuru3IGhtnk8NkCzEnAziD1OtEnEn0lsOePXsQEBAAnU6H/fv3AwAsFkuzAlwQBIcV6GpKDE3U8p2VuMJYbrvUSj3ZQg6tl87qTMtVSZsMip4cDh48CKPRiNjYWNy4cQO3b9/GlStX4O7ubv2M0WiEVqsVOzSnq6ysRGRkJB5++GFoNBrU19dDq9UiLS0N/v7+UoenOErtr5a6QJYTuSXLjujM1v9K6gYVPTns2LHD+u/9+/fjiy++wGuvvYaIiAiUl5cjMDAQeXl5SEhIEDs0K2fuqaLVapGTk2N9nZ6ejs2bN2PLFs7K6Cj2V7sGpSbLjrZc5bDosCNkkaY8PT2Rnp6OlJQUREdHY9CgQYiMjJQ6LFGMGzcO33zzDf7+97/jmWeewbRp0xAZGYnTp08DaEym06ZNQ1xcHNasWQMAuHDhAp555hnEx8cjMTERly9flvAbSEfp/dWkbB1tucph0WFHSLoILj4+HvHx8QAAnU6H3NxcKcMRXX19PT799FOMHDkSu3fvxjvvvANfX1/s3bsXWVlZ2LZtG7Zv346jR4/C3d0dq1atgl6vx/vvv4+kpCRERUXhwIEDKC0txUMPPST11xGdkvurSYCSGnQAABAPSURBVPk62nJVWjcoV0j/QqyHoRsMBsTGxgIA6urq8Lvf/Q7Lli1Dt27dUFBQgEuXLuGLL76Am5sb3N3dMWrUKEyfPh2PP/44kpKS0K9fP4SFhWH9+vU4evQoJk+ejEmTJjkkNqVRcn81KV9HpyUrrRuUyUFk9445AEBNTQ0SEhIwbdo0jBkzBkOGDEF2djYA4K233kJpaSn+7//+D/Pnz0dGRgYiIyMxatQoHDlyBDt37kRhYSE2bNggxdeRnFL7q0n5OtpyldMal/ZgcviFlA/5uHz5MjQaDRYuXAhBEPDyyy/DbDbj2rVrmDVrFvbu3YtRo0bhp59+QllZGT788ENMnToVM2bMwMMPP4y0tDRR4iSiX3W05aq0blAmBxkYOnQoHnnkEURFRUGj0SA0NBSnTp2Cr68vnn32WUyfPh3e3t4YOHAgEhISMGbMGKxatQrbtm2Dh4cH1q1bJ/VXEI2S5omT6+tIy1Vp3aAaQRAEqYPoisrKSjz++OM4fPgwAgOb99mdP38ejzzySId/pis8HrCz313OWponDvxa85LbPHEiOWur7ATYcmiRkpOCK1PaPHEiJWM1ixRDafPEiZSMyYEUQ2nzxImUjMmBFEMOD6chUgsmB1IMbpdBJB4mB1IMOT0LgcjVcbYSKYbS5okTKRmTg4jufp7D3d555x0EBATYfP6jjz4CACQmJmLIkCEoKysTJU4543YZROJgcriLGKtvW9pbqTWJiYkOOScRUUcxOfxCyqc0Xbx4Ef/+7/+O27dv49q1a1iwYAESExPxl7/8BQCQkpLilPMSEbWGyeEXYq2+vXvLbgCIiYmBXq/HokWLoNPpUFFRgWnTprHVQESSYnL4hVgPq2+pW8lsNuPo0aPYvn07Ll68iNu3b3f5PEREXcHpHb+QcvXtkiVLcOjQITz88MNYsmSJ085D1BJBsECvz0ZJyWgcP94PJSWjoddnQxAsUodGEmLL4RdSPqXp+PHj+Pvf/45+/fpZH/JjNpuddj6iJlKOtZG88bf+CylX36akpGDmzJmIjo7GqVOn0L9/f1RWcp8gcr72jLWROknScnjzzTfx6aefQqPRYPr06UhKSkJxcTHS0tJgMpkQFRWF1NRUUWMS4ylNgYGBKCgosDmelJSEpKQkm+N3z1LiGgdyBrHG2kh5RE8OX3zxBT7//HPk5uaioaEB0dHR0Ol0WLlyJXbt2oWAgAAkJyejqKgIYWFhosXF1bekRtzpllojenIYO3YsPvjgA3Tr1g16vR5msxk3b95EcHAwgoIad92MiYlBfn6+qMkB4OpbUh8px9pI3iSpDnt4eGDr1q2YMmUKdDodDAYD/Pz8rO9rtVro9XopQiNSFe50S62RrK/khRdewIkTJ/Djjz/i8uXL0Gg01vcEQWj2uisU/ojsTlHjd6bO4U631BrRk8O3336L8+fPAwC8vb0RERGBkydPwmg0Wj9jNBqh1Wq7fC4vLy9UVVWpqrAUBAFVVVXw8vKSOhRSgKaxtiFDsuDjEwIPj37w8QnBkCFZnMaqcqKPOVRWVmLr1q3WHUcPHz6MGTNmYPPmzSgvL0dgYCDy8vKQkJDQ5XMFBgaisrKyWeJRAy8vLwQGsq+Y2odjbdQS0ZNDWFgYzp49i7i4OLi7uyMiIgJTpkyBr68vUlJSYDKZEBYWhsjIyC6fy8PDAwMHDnRA1ERE6iLJOoeUlBSbnUZ1Oh1yc3OlCIeIiO7BDkUiIrKh+L2VmvYg+umnnySOhIhIOZrKzNb2cVN8cmgabJ41a5bEkRARKY/RaERwcLDNcY2g8HmetbW1OHfuHPz8/ODu7i51OEREimA2m2E0GjF8+PAWp74rPjkQEZHjcUCaiIhsMDkQEZENJgciIrLB5EBERDaYHIiIyAaTAxER2WByICIiG6pKDm+++Saio6MxZcoU7NixAwBQXFyMmJgYREREIDMzU+IIW7Zp0yasWLECgLzjnT17NqZMmYLY2FjExsbizJkzso23oKAA8fHxiIqKwoYNGwDI99ru2bPHek1jY2MREhKC9evXyzZeAMjJycGUKVMwZcoUbNq0CYB8r29WVhaefPJJxMTE4O233wYgv1irq6sxdepUVFY2PtO7tfjOnz+P+Ph4PPnkk1i1ahUaGho6f1JBJU6ePCnMmDFDqK+vF+7cuSNMmjRJOH/+vBAWFiZ8//33Qn19vTBv3jyhsLBQ6lCbKS4uFsaNGycsX75cuHPnjmzjtVgsQmhoqFBfX289Jtd4v//+eyE0NFT48ccfhbq6OiExMVEoLCyUZaz3unjxovD73/9e+OGHH2Qb7+3bt4UxY8YIVVVVQn19vTB9+nTh8OHDsoz3+PHjwtSpU4Vbt24JDQ0NQnJyspCTkyOrWEtLS4WpU6cKw4YNEyoqKtq8r6ZMmSJ89dVXgiAIwiuvvCJkZ2d3+ryqaTmMHTsWH3zwAbp164aqqiqYzWbcvHkTwcHBCAoKQrdu3RATE4P8/HypQ7W6fv06MjMzsXDhQgDA2bNnZRvvd999BwCYN28epk2bhr/97W+yjffQoUOIjo6Gv78/PDw8kJmZCW9vb1nGeq9169YhNTUVFRUVso3XbDbDYrHgzp07aGhoQENDA3x8fGQZ79dff43Q0FD4+PjA3d0dEyZMwJ49e2QV6yeffIK1a9dan47Z2n115coV1NbWYuTIkQCA+Pj4LsWtmuQAND78Z+vWrZgyZQp0Oh0MBgP8/Pys72u1Wuj1egkjbG7NmjVITU3FfffdBwCyjvfmzZvQ6XTYtm0bdu7cid27d+OHH36QZbzl5eUwm81YuHAhYmNj8eGHH8r62jYpLi5GbW0toqKiZB2vj48PXnzxRURFRSEsLAz9+/eXbbzDhg3DsWPHcP36dZhMJhQUFOD06dOyinXjxo0YPXq09XVr1/Le435+fl2KW1XJAQBeeOEFnDhxAj/++CMuX74MjUZjfU8QhGavpbRnzx4EBARAp9NZj1ksFtnGO2rUKGzevBm9evWCr68vpk+fjq1bt8oyXrPZjBMnTuD111/Hxx9/jLNnz6KiokKWsd5t9+7dSEpKAiDvv4ULFy5g3759OHLkCI4ePQo3NzfZ3ms6nQ7x8fGYPXs25s+fj5CQEDQ0NMgy1iat/e4d/Teh+C272+vbb79FXV0dHnnkEXh7eyMiIgL5+fnNdnI1Go3WppvUDh48CKPRiNjYWNy4cQO3b9/GlStXZBtvSUkJ6uvrrclMEAT079+/2fO75RJv3759odPp4OvrCwB44oknZP23AAB1dXX48ssvkZ6eDgDw9/eX5bUFgGPHjkGn06FPnz4AGrs3/uM//kOW17e6uhoRERHWpPvee+9h7Nixsr22QOu/+3uPX716tUtxq6blUFlZidWrV6Ourg51dXU4fPgwZsyYgUuXLlm7GfLy8jBx4kSpQwUA7NixA3l5ecjJycELL7yAyZMn47333pNtvLdu3cLmzZthMplQXV2NAwcOYOnSpbKMd9KkSTh27Bhu3rwJs9mMo0ePIjIyUpaxNikrK8NDDz2EHj16AABGjBgh23iHDh2K4uJi3L59G4IgoKCgQLbxVlZWYtGiRWhoaMCtW7ewd+9eLFmyRJaxNmntWvbv3x+enp44deoUgMYZY12JWzUth7CwMJw9exZxcXFwd3dHREQEpkyZAl9fX6SkpMBkMiEsLAyRkZFSh9oqT09PpKenyzLeSZMm4cyZM4iLi4PFYsHMmTMxatQoWcY7YsQIzJ8/HzNnzkR9fT3Gjx+PxMREDBo0SHaxNqmoqIC/v7/1tZz/FkJDQ/H1118jPj4eHh4eePTRR5GSkoLx48fLLt6hQ4ciIiIC06ZNg9lsxty5cxESEiLbawu0/bvPyMjA6tWrUV1djWHDhuG5557r9Hn4PAciIrKhmm4lIiJqPyYHIiKyweRAREQ2mByIiMgGkwMREdlQzVRWUo8hQ4Zg8ODBcHNrXvfZtm0bAgMDRY1FEASsWLECgwcPxh/+8AdRz03UFUwO5JLef/996wpoqXz77bd47bXXcPbsWQwePFjSWIg6ismBVOXAgQPYtm0bcnJyoNFokJCQgOTkZEybNg2vv/46zpw5g5qaGgiCgA0bNiAkJAQrVqyAl5cXLl68iKqqKkyePBm9e/fGkSNHYDQasWHDhmZ7YDXJzs7G008/jQcffLDVeGpqavDKK6+gvLwcbm5uGDZsGNavXw83Nzfs3bsXO3bsgJubGx544AFs2rQJAQEB+Pjjj7Fr1y64ubmhb9++ePXVVzFw4ECsWLEC169fR0VFBcLDw/Hiiy8iIyMDX375JcxmM/7pn/4Jq1evho+PjzMvMbkIJgdySXPmzGnWrRQYGIht27bhqaeewrFjx/DGG2+grq4Oo0ePRlxcHL766isYDAZ8/PHHcHNzQ1ZWFt59912EhIQAaNzaOTs7G9evX0doaChWr16N3bt34/3338e7777bYnJYs2YNAOD48eOtxnno0CHU1NQgJycHZrMZa9euRUVFBe7cuYOMjAwcOHAAAQEB2LlzJ95++21ERUXhvffew8cffwxfX1/s378fixcvxv/8z/8AAGpra63//utf/wp3d3fs378fGo0GW7ZsQUZGBtatW+eoy0wujMmBXFJb3UqvvfYaYmNj4eXlhf379wNo3FX2/vvvx+7du1FRUYGTJ0+iZ8+e1v8zadIkeHh4wM/PDz169MCECRMAAAMGDMD169c7HWdISAgyMzMxe/Zs/Mu//AvmzJmD4OBg7NixA6GhoQgICAAAzJ07FwCwefNmREdHW79bfHw8Nm7caH1CWFMyA4DCwkLcunULxcXFAID6+nrrZnhE9jA5kOpUVVXBZDKhrq4OBoMBQUFBKCwsxMaNG5GUlITHH38cgwYNQm5urvX/dO/evdnP6NbNMbdOUFAQDh06hJMnT+Lzzz9HUlIS1q9fD3d392bbLdfW1uLKlSuwWCw2P0MQBOvjIJs25gMat3ZeuXIlwsLCADR2YZlMJofETa6PU1lJVerr67F06VK8+OKL+NOf/oTU1FTU19fj+PHjmDRpEmbOnInhw4fjs88+g9lsdno8H374IV555RWEhobipZdesm5aN27cOJw4cQIGgwFA47Mc3njjDUyYMAEHDx7EtWvXAAD79u1D7969ERwcbPOzQ0NDkZ2djbq6OlgsFrz66qvYsmWL078TuQa2HMgl3TvmAABLly7F559/jr59++Lpp58GAHz22WfIzMzEjBkzsGzZMsTExKChoQHjx4/H//7v/7ZYU3ekuLg4fPHFF4iOjoa3tzcCAgIwe/Zs3H///XjppZcwf/58AI1P9Xr99dfRr18/zJ07F3PmzIHFYoGvry+2b99u810BYNGiRdi0aROeeuopmM1mPPLII1ixYoVTvw+5Du7KSkRENtitRERENpgciIjIBpMDERHZYHIgIiIbTA5ERGSDyYGIiGwwORARkQ0mByIisvH/AavzENZrt8VWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotData(data, 'Exam 1 score', 'Exam 2 score', 'Pass', 'Fail')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 自定义逻辑回归算法\n",
    "#### $$ h_{\\theta}(x) = g(\\theta^{T}x)$$\n",
    "#### $$ g(z)=\\frac{1}{1+e^{−z}} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义 sigmod 函数\n",
    "def sigmod(z):\n",
    "    return 1/(1 + np.exp(-z))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 对数损失函数\n",
    "#### $$ J(\\theta) = \\frac{1}{m}\\sum_{i=1}^{m}\\big[-y^{(i)}\\, log\\,( h_\\theta\\,(x^{(i)}))-(1-y^{(i)})\\,log\\,(1-h_\\theta(x^{(i)}))\\big]$$\n",
    "## 向量化的损失函数(矩阵形式)\n",
    "#### $$ J(\\theta) = -\\frac{1}{m}\\big((\\,log\\,(g(X\\theta))^Ty+(\\,log\\,(1-g(X\\theta))^T(1-y)\\big)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义损失函数\n",
    "def costFunc(theta, X, Y):\n",
    "    m = Y.size\n",
    "    h = sigmod(X.dot(theta))\n",
    "    \n",
    "    J = -(1.0/m)*(np.log(h).T.dot(Y) + np.log(1 - h).T.dot(1-Y))\n",
    "    \n",
    "    if np.isnan(J[0]):\n",
    "        return np.inf\n",
    "    return J[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 求偏导(梯度)\n",
    "\n",
    "#### $$ \\frac{\\delta J(\\theta)}{\\delta\\theta_{j}} = \\frac{1}{m}\\sum_{i=1}^{m} ( h_\\theta (x^{(i)})-y^{(i)})x^{(i)}_{j} $$ \n",
    "## 向量化的偏导(梯度)\n",
    "#### $$ \\frac{\\delta J(\\theta)}{\\delta\\theta_{j}} = \\frac{1}{m} X^T(g(X\\theta)-y)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 梯度下降算法\n",
    "def gradient(theta, X, Y):\n",
    "    m = Y.size\n",
    "    h = sigmod(X.dot(theta.reshape(-1, 1)))\n",
    "    \n",
    "    grad = (1.0/m)*X.T.dot(h-Y)\n",
    "    return grad.flatten()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 初始位置的损失函数值大小（预测值和真实值差异大小）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "损失函数初始值=> 0.6931471805599452\n",
      "下降梯度初始值=> [ -0.1        -12.00921659 -11.26284221]\n"
     ]
    }
   ],
   "source": [
    "initial_theta = np.zeros(X.shape[1])\n",
    "cost = costFunc(initial_theta, X, Y)\n",
    "grad = gradient(initial_theta, X, Y)\n",
    "print(\"损失函数初始值=>\", cost)\n",
    "print(\"下降梯度初始值=>\", grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 使用优化函数求解损失函数最小值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-27-51a1dcc63605>:6: RuntimeWarning: divide by zero encountered in log\n",
      "  J = -(1.0/m)*(np.log(h).T.dot(Y) + np.log(1 - h).T.dot(1-Y))\n",
      "<ipython-input-27-51a1dcc63605>:6: RuntimeWarning: divide by zero encountered in log\n",
      "  J = -(1.0/m)*(np.log(h).T.dot(Y) + np.log(1 - h).T.dot(1-Y))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "      fun: 0.20349770158950983\n",
       " hess_inv: array([[ 2.85339493e+03, -2.32908823e+01, -2.27416470e+01],\n",
       "       [-2.32908823e+01,  2.04489131e-01,  1.72969525e-01],\n",
       "       [-2.27416470e+01,  1.72969525e-01,  1.96170322e-01]])\n",
       "      jac: array([-2.68557631e-09,  4.36433478e-07, -1.39671758e-06])\n",
       "  message: 'Optimization terminated successfully.'\n",
       "     nfev: 33\n",
       "      nit: 25\n",
       "     njev: 29\n",
       "   status: 0\n",
       "  success: True\n",
       "        x: array([-25.16131634,   0.2062316 ,   0.20147143])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = minimize(costFunc, initial_theta, args=(X, Y), jac=gradient, options={'maxiter': 400})\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 自定义预测函数，预测是否通过\n",
    "def predict(theta, X, threshold=0.5):\n",
    "    p = sigmod(X.dot(theta.T)) >= threshold\n",
    "    return p.astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "成绩一45分，成绩二85分通过测试的概率=> 0.7762903249331015\n",
      "成绩一45分，成绩二85分是否通过测试=> 1\n"
     ]
    }
   ],
   "source": [
    "p = sigmod(np.array([1, 45, 85]).dot(res.x.T))\n",
    "passOrNot = predict(res.x.T, np.array([1, 45, 85]))\n",
    "print(\"成绩一45分，成绩二85分通过测试的概率=>\", p)\n",
    "print(\"成绩一45分，成绩二85分是否通过测试=>\", passOrNot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 非线性决策边界"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
